{
  kgs indexer

  @Author    csyap
  @Version   2010.11.13 first version

  TODO , test tempfile, merging.
         write index time metadata
         error report system

  TODO test tipitaka         
}
unit kgs_indexer;

interface
uses classes,sysutils,kgs_classes,kgs_tokenizer,kgs_postings,tempfile;
const
  DEFAULTMAXTOKENPERBLOCK=64*1024;
  MAXWIDECHAR=65536;
type
  TKGSIndexer = Class(TPersistent)
    protected
      fdb:TKGSDatabase;
      ftablename:widestring;
      tokenizer:TTokenizer;
      fMaxtokenperblock:integer;
      ftokencount:int64;
      fva:int64;

      widecharmap: array [0..MAXWIDECHAR-1] of TVintList_wo;
      frowid:int64;
      findexjob:widestring;
      procedure buildinverted(s:widestring);
    public
      onindexprogress:TNotifyEvent;
      onindexjob:TNotifyEvent;

      progressmax,progressnow:integer;

      constructor create(db:TKGSDatabase);
      destructor destroy;override;
      procedure build(tablename:widestring; tknr:string=DEFAULTTOKENIZER);virtual;
      procedure addwidechartoken(c:widechar);

      procedure writePostings;
      procedure putinverted(v:TvintList;token:widestring);
      procedure writemeta;
      procedure setindexJob(job:widestring);

      property tokencount:int64 read ftokencount;
      property rowid:int64 read frowid;
      property va:int64 read fva;
      property Maxtokenperblock:integer read fMaxtokenperblock;
      property indexjob:widestring read findexjob write setIndexJob ;
  end;

implementation

{ TKGSIndexer }
procedure TKGSIndexer.addwidechartoken(c:widechar);
var
  v:TVintList_wo;
  u:cardinal;
begin
  u:=cardinal(c);
  if not assigned(widecharmap[u]) then widecharmap[u]:=TVintList_wo.create;
  v:=widecharmap[u];
  v.add(fva);
end;

procedure TKGSIndexer.buildinverted(s:widestring);
var
  indexedtoken:integer;
begin
    indexedtoken:=0;
    fva:=fMaxtokenperblock*frowid;
    tokenizer.setup(pwidechar(s));
    if assigned(onindexprogress) then onindexprogress(self);
    while tokenizer.next  do begin
      inc(indexedtoken);
      if indexedtoken > fMaxtokenperblock then begin
        raise exception.create(
          format('block to big, try to increase Maxtokenperblock row:%d',[frowid]));
        break;
      end;
      case tokenizer.token.tokentype of
        ttCJK : begin
          addwidechartoken(tokenizer.rawtoken[0]);
        end;
      end;
      inc(fva);
    end;
end;

procedure TKGSIndexer.putinverted(v:TvintList;token:widestring);
var
  ms_offset,ms_count:Tmemorystream;
begin
  ms_offset:=TmemoryStream.create;
  ms_count:=TmemoryStream.create;
  v.packmilestone(ms_offset,ms_count);
  fdb.putinverted(ftablename,token,v.data,v.size,ms_offset,ms_count);
  ms_offset.free;
  ms_count.free;
end;

procedure TKGSIndexer.writePostings;
var
  v:TVintList;
  token:widestring;
  i:integer;
begin
  fdb.begintransaction;
  for i:=0 to MAXWIDECHAR-1 do begin
    v:=TVintList.create;
    if widecharmap[i]=nil then continue;
    v.copyfrom(widecharmap[i]);
    token:=widechar(i);
    putinverted(v,token);
    widecharmap[i].free;
    v.free;
  end;
  fdb.commit;
end;

procedure TKGSIndexer.build(tablename:widestring; tknr:string=DEFAULTTOKENIZER);
var
  rowid:int64;
  id,body:widestring;
  sl:tstringlist;
  tokenizerclass:TTokenizerClass;
begin
  ftablename:=tablename;
  fdb.createinverted(tablename);
  fdb.select(tablename);
  
  inittemp(tablename+'.tmp');

  tokenizerclass:=TTokenizerClass(FindClass(tknr));
  if not assigned(tokenizerclass) then begin
    raise exception.create('error finding class '+tknr);
  end;
  if assigned(tokenizer) then freeandnil(tokenizer);
  tokenizer:=Tokenizerclass.create;



  progressmax:=fdb.getrowcount(tablename);
  progressnow:=0;
  indexjob:='indexing';
  //build inverted index for each row
  while fdb.fetch(frowid,id,body) do begin
    inc(progressnow);
    buildinverted(body);
  end;

  WritePostings;

  WriteMeta;

  fdb.doneinverted(tablename);
  finalizetemp();
end;

procedure TKGSIndexer.writemeta;
begin
//write DEFAULTMAXTOKENPERBLOCK
end;
constructor TKGSIndexer.create(db: TKGSDatabase);
begin
  fdb:=db;
  fMaxtokenperblock:=DEFAULTMAXTOKENPERBLOCK;
  fillchar(widecharmap,sizeof(widecharmap),0);
end;

destructor TKGSIndexer.destroy;
begin
  if assigned(tokenizer) then freeandnil(tokenizer);
  inherited;
end;

procedure TKGSIndexer.setindexJob(job: widestring);
begin
  findexjob:=job;
  if assigned(onindexjob) then onindexjob(self);
end;

end.
